In [1]:
%matplotlib inline
import pandas as pd
from tools import ready_data
# read csv file and reoder data as input files, voter1, voter2, ...
file1 = 'results-3AFT28WXLF2Y43L0GI7MY3EYIU1IO5.csv'
file2 = 'results-3P4ZBJFX2V3MZLSC6WSBC0ES548FWN.csv'
In [2]:
## Correlation p-values for Pearson's r
from scipy.stats import pearsonr
## Adapted from: https://stackoverflow.com/questions/25571882/pandas-columns-correlation-with-statistical-significance/45507587#45507587
def corr_pvalues(df):
# df = df.dropna()._get_numeric_data()
dfcols = pd.DataFrame(columns=df.columns)
pvalues = dfcols.transpose().join(dfcols, how='outer')
for r in df.columns:
for c in df.columns:
pvalues[r][c] = pearsonr(df[r], df[c])[1]
return pvalues
Boxplot of likert score¶
Art_freeform_AP_03¶
In [3]:
df1, df2, name1, header1 = ready_data(file1)
print("The boxplot of sketch %s"%name1)
# drop the last 7 rows, since the metrics are not necessary in boxplot
df_2_refine = df2.drop(df2.tail(7).index)
df_2_refine = df_2_refine.sort_index(axis=1)
boxplot1 = df_2_refine.boxplot(grid=False, rot=90, showmeans = True)
boxplot1.figure.savefig('Art_freeform_AP_03-boxplot.pdf')
boxplot1
Out[3]:
In [4]:
df_corr = df2.copy()
df_corr.loc[ df_corr.voter.str.match('voter_[0-9]+'), 'voter' ] = 'human'
df_corr.loc[ df_corr.voter == 'voter_chamfer', 'voter' ] = 'chamfer'
df_corr
Out[4]:
Correlation of mean Likert scores to computed metrics¶
In [5]:
df_corr_agg = df_corr.groupby('voter').mean().T
In [6]:
df_corr_agg.corr()
Out[6]:
In [7]:
# print( "Correlation coefficient, p value" )
# print( pearsonr( df_corr_agg.chamfer, df_corr_agg.human ) )
corr_pvalues( df_corr_agg )
Out[7]:
Correlation without MasteringSketching Likert scores¶
In [8]:
df_corr_agg = df_corr.groupby('voter').mean().drop(columns = ["ALG: MasteringSketching"]).T
Correlation is not defined (NaN) for constant data. F-score with a very high threshold (36/1000) find perfect similarity (a constant 1) in all cases except MasteringSketching. With MasteringSketching excluded, correlation is no longer defined.
In [9]:
df_corr_agg.corr()
Out[9]:
In [10]:
corr_pvalues( df_corr_agg )
Out[10]:
Ind_architecture_TU_02¶
In [11]:
df3, df4, name2, header2 = ready_data(file2)
print("The boxplot of sketch %s"%name2)
df_4_refine = df4.drop(df4.tail(7).index)
df_4_refine = df_4_refine.sort_index(axis=1)
boxplot2 = df_4_refine.boxplot(grid=False, rot=90, showmeans = True)
boxplot2.figure.savefig('Ind_architecture_TU_02-boxplot.pdf')
boxplot2
Out[11]:
In [12]:
df_corr = df4.copy()
df_corr.loc[ df_corr.voter.str.match('voter_[0-9]+'), 'voter' ] = 'human'
df_corr.loc[ df_corr.voter == 'voter_chamfer', 'voter' ] = 'chamfer'
df_corr
Out[12]:
Correlation of mean Likert scores to computed metrics¶
In [13]:
df_corr_agg = df_corr.groupby('voter').mean().T
In [14]:
df_corr_agg.corr()
Out[14]:
In [15]:
corr_pvalues( df_corr_agg )
Out[15]:
In [16]:
print( pearsonr( df_corr_agg.chamfer, df_corr_agg.human ) )
Correlation without MasteringSketching Likert scores¶
In [17]:
df_corr_agg = df_corr.groupby('voter').mean().drop(columns = ["ALG: MasteringSketching"]).T
In [18]:
df_corr_agg.corr()
Out[18]:
In [19]:
corr_pvalues( df_corr_agg )
Out[19]:
Does the rank of algorithms stay consistent?¶
A low p-value means that the two rankings are the same.
In [20]:
df_ind = df4[ df4.voter.str.match('voter_[0-9]+') ].drop( columns=['voter'] ).mean()
df_ind
Out[20]:
In [21]:
df_art = df2[ df4.voter.str.match('voter_[0-9]+') ].drop( columns=['voter'] ).mean()
df_art
Out[21]:
In [22]:
likert_averages = pd.DataFrame({'Art_freeform_AP_03': df_art, 'Ind_architecture_TU_02': df_ind })
likert_averages
Out[22]:
In [23]:
import scipy.stats
scipy.stats.wilcoxon( likert_averages.Art_freeform_AP_03, likert_averages.Ind_architecture_TU_02 )
# scipy.stats.wilcoxon( likert_averages.Art_freeform_AP_03, likert_averages.Art_freeform_AP_03 + 0.01 )
Out[23]: